{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "PyTorch: Tensor和autograd\n",
    "-------------------------------\n",
    "\n",
    "还是和前面一样实现一个全连接的网络，只有一个隐层而且没有bias，使用欧氏距离作为损失函数。\n",
    "\n",
    "这个实现使用PyTorch的Tensor来计算前向阶段，然后使用PyTorch的autograd来自动帮我们反向计算梯度。\n",
    "\n",
    "\n",
    "PyTorch的Tensor代表了计算图中的一个节点。如果``x``是一个Tensor并且``x.requires_grad=True``，\n",
    "那么``x.grad``这个Tensor会保存某个scalar(通常是loss)对``x``的梯度。\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 42577276.0\n",
      "1 50359256.0\n",
      "2 65796516.0\n",
      "3 67674760.0\n",
      "4 42845276.0\n",
      "5 15258689.0\n",
      "6 4446020.5\n",
      "7 2015284.25\n",
      "8 1388696.375\n",
      "9 1098846.25\n",
      "10 901679.8125\n",
      "11 750083.75\n",
      "12 629551.9375\n",
      "13 532398.6875\n",
      "14 453381.90625\n",
      "15 388429.625\n",
      "16 334606.5\n",
      "17 289705.4375\n",
      "18 252044.1875\n",
      "19 220220.96875\n",
      "20 193169.21875\n",
      "21 170088.75\n",
      "22 150274.703125\n",
      "23 133194.75\n",
      "24 118415.890625\n",
      "25 105580.4921875\n",
      "26 94393.828125\n",
      "27 84614.8984375\n",
      "28 76035.4375\n",
      "29 68481.7734375\n",
      "30 61815.71484375\n",
      "31 55911.49609375\n",
      "32 50669.3203125\n",
      "33 45999.05859375\n",
      "34 41832.07421875\n",
      "35 38103.828125\n",
      "36 34761.57421875\n",
      "37 31763.423828125\n",
      "38 29069.8984375\n",
      "39 26642.462890625\n",
      "40 24449.857421875\n",
      "41 22465.3515625\n",
      "42 20666.79296875\n",
      "43 19033.759765625\n",
      "44 17548.796875\n",
      "45 16196.2939453125\n",
      "46 14963.87890625\n",
      "47 13838.119140625\n",
      "48 12808.517578125\n",
      "49 11865.458984375\n",
      "50 11001.185546875\n",
      "51 10208.166015625\n",
      "52 9479.0048828125\n",
      "53 8808.4482421875\n",
      "54 8191.31982421875\n",
      "55 7622.75634765625\n",
      "56 7098.291015625\n",
      "57 6613.82666015625\n",
      "58 6166.24951171875\n",
      "59 5752.46044921875\n",
      "60 5369.3828125\n",
      "61 5014.4453125\n",
      "62 4685.76123046875\n",
      "63 4380.861328125\n",
      "64 4097.64306640625\n",
      "65 3834.5703125\n",
      "66 3590.062744140625\n",
      "67 3362.572021484375\n",
      "68 3150.78564453125\n",
      "69 2953.6435546875\n",
      "70 2770.001708984375\n",
      "71 2598.693603515625\n",
      "72 2438.92431640625\n",
      "73 2289.849609375\n",
      "74 2150.6181640625\n",
      "75 2020.529541015625\n",
      "76 1898.9683837890625\n",
      "77 1785.3828125\n",
      "78 1679.078125\n",
      "79 1579.5634765625\n",
      "80 1486.43115234375\n",
      "81 1399.1904296875\n",
      "82 1317.445556640625\n",
      "83 1240.822998046875\n",
      "84 1168.9993896484375\n",
      "85 1101.61181640625\n",
      "86 1038.38525390625\n",
      "87 979.05859375\n",
      "88 923.3375244140625\n",
      "89 870.98681640625\n",
      "90 821.9129638671875\n",
      "91 775.9140625\n",
      "92 732.68115234375\n",
      "93 692.020751953125\n",
      "94 653.7692260742188\n",
      "95 617.8558349609375\n",
      "96 584.1307373046875\n",
      "97 552.3668823242188\n",
      "98 522.4515991210938\n",
      "99 494.2625732421875\n",
      "100 467.6861572265625\n",
      "101 442.6285705566406\n",
      "102 419.0074157714844\n",
      "103 396.7127380371094\n",
      "104 375.67584228515625\n",
      "105 355.818115234375\n",
      "106 337.0773620605469\n",
      "107 319.37432861328125\n",
      "108 302.660400390625\n",
      "109 286.86981201171875\n",
      "110 271.957275390625\n",
      "111 257.8547668457031\n",
      "112 244.52357482910156\n",
      "113 231.92262268066406\n",
      "114 220.00865173339844\n",
      "115 208.74179077148438\n",
      "116 198.08135986328125\n",
      "117 187.99842834472656\n",
      "118 178.45091247558594\n",
      "119 169.4109344482422\n",
      "120 160.84921264648438\n",
      "121 152.74940490722656\n",
      "122 145.07652282714844\n",
      "123 137.80374145507812\n",
      "124 130.9171600341797\n",
      "125 124.38938903808594\n",
      "126 118.20165252685547\n",
      "127 112.33438873291016\n",
      "128 106.77377319335938\n",
      "129 101.4993896484375\n",
      "130 96.49832153320312\n",
      "131 91.75255584716797\n",
      "132 87.25187683105469\n",
      "133 82.98217010498047\n",
      "134 78.92718505859375\n",
      "135 75.07981872558594\n",
      "136 71.42893981933594\n",
      "137 67.96247863769531\n",
      "138 64.66929626464844\n",
      "139 61.541812896728516\n",
      "140 58.573829650878906\n",
      "141 55.75383377075195\n",
      "142 53.074195861816406\n",
      "143 50.52786636352539\n",
      "144 48.10978317260742\n",
      "145 45.81015396118164\n",
      "146 43.62591552734375\n",
      "147 41.547393798828125\n",
      "148 39.57353210449219\n",
      "149 37.69597625732422\n",
      "150 35.91054916381836\n",
      "151 34.21224594116211\n",
      "152 32.59809112548828\n",
      "153 31.061721801757812\n",
      "154 29.60047721862793\n",
      "155 28.21023178100586\n",
      "156 26.887786865234375\n",
      "157 25.629621505737305\n",
      "158 24.43067169189453\n",
      "159 23.290353775024414\n",
      "160 22.204925537109375\n",
      "161 21.171842575073242\n",
      "162 20.18770408630371\n",
      "163 19.251455307006836\n",
      "164 18.359827041625977\n",
      "165 17.510456085205078\n",
      "166 16.70132064819336\n",
      "167 15.931001663208008\n",
      "168 15.197333335876465\n",
      "169 14.49821949005127\n",
      "170 13.831990242004395\n",
      "171 13.197452545166016\n",
      "172 12.592574119567871\n",
      "173 12.016409873962402\n",
      "174 11.467093467712402\n",
      "175 10.943482398986816\n",
      "176 10.444195747375488\n",
      "177 9.96908950805664\n",
      "178 9.515769004821777\n",
      "179 9.083292961120605\n",
      "180 8.671213150024414\n",
      "181 8.278311729431152\n",
      "182 7.903132438659668\n",
      "183 7.546097755432129\n",
      "184 7.205053806304932\n",
      "185 6.879815101623535\n",
      "186 6.570107936859131\n",
      "187 6.2741289138793945\n",
      "188 5.991806983947754\n",
      "189 5.722585201263428\n",
      "190 5.466098785400391\n",
      "191 5.2212347984313965\n",
      "192 4.987483024597168\n",
      "193 4.764040946960449\n",
      "194 4.551290988922119\n",
      "195 4.347972393035889\n",
      "196 4.153834819793701\n",
      "197 3.968749523162842\n",
      "198 3.7918059825897217\n",
      "199 3.6232690811157227\n",
      "200 3.4623422622680664\n",
      "201 3.308492422103882\n",
      "202 3.1617496013641357\n",
      "203 3.021618366241455\n",
      "204 2.887803316116333\n",
      "205 2.75993275642395\n",
      "206 2.6378984451293945\n",
      "207 2.5213308334350586\n",
      "208 2.4100470542907715\n",
      "209 2.3038549423217773\n",
      "210 2.202146291732788\n",
      "211 2.1052846908569336\n",
      "212 2.0127623081207275\n",
      "213 1.9241505861282349\n",
      "214 1.839674949645996\n",
      "215 1.7589720487594604\n",
      "216 1.6817970275878906\n",
      "217 1.608252763748169\n",
      "218 1.5377006530761719\n",
      "219 1.47026526927948\n",
      "220 1.406042456626892\n",
      "221 1.344491720199585\n",
      "222 1.2857954502105713\n",
      "223 1.229628086090088\n",
      "224 1.1759642362594604\n",
      "225 1.1246857643127441\n",
      "226 1.0757228136062622\n",
      "227 1.028974175453186\n",
      "228 0.9842228889465332\n",
      "229 0.9414544701576233\n",
      "230 0.9004974961280823\n",
      "231 0.8614296317100525\n",
      "232 0.8239631652832031\n",
      "233 0.7883274555206299\n",
      "234 0.7541966438293457\n",
      "235 0.7215466499328613\n",
      "236 0.6903102397918701\n",
      "237 0.6604781746864319\n",
      "238 0.6318474411964417\n",
      "239 0.6045924425125122\n",
      "240 0.5783928036689758\n",
      "241 0.5534623265266418\n",
      "242 0.5295946598052979\n",
      "243 0.5067638754844666\n",
      "244 0.48489803075790405\n",
      "245 0.46407654881477356\n",
      "246 0.444137841463089\n",
      "247 0.4250060021877289\n",
      "248 0.40667933225631714\n",
      "249 0.3892454206943512\n",
      "250 0.3725389540195465\n",
      "251 0.3565327823162079\n",
      "252 0.34125569462776184\n",
      "253 0.32660189270973206\n",
      "254 0.3126069903373718\n",
      "255 0.2992696464061737\n",
      "256 0.2864656150341034\n",
      "257 0.2741631269454956\n",
      "258 0.2624094486236572\n",
      "259 0.25119462609291077\n",
      "260 0.24045389890670776\n",
      "261 0.23017586767673492\n",
      "262 0.2203713059425354\n",
      "263 0.21095426380634308\n",
      "264 0.20194299519062042\n",
      "265 0.1933317631483078\n",
      "266 0.1850835680961609\n",
      "267 0.1771990954875946\n",
      "268 0.16963402926921844\n",
      "269 0.16243264079093933\n",
      "270 0.15555213391780853\n",
      "271 0.14894460141658783\n",
      "272 0.14259064197540283\n",
      "273 0.13652533292770386\n",
      "274 0.13074861466884613\n",
      "275 0.12517690658569336\n",
      "276 0.11986954510211945\n",
      "277 0.11477436870336533\n",
      "278 0.109909288585186\n",
      "279 0.10525605827569962\n",
      "280 0.10076799243688583\n",
      "281 0.096539206802845\n",
      "282 0.09243905544281006\n",
      "283 0.08850731700658798\n",
      "284 0.0847858339548111\n",
      "285 0.08121336996555328\n",
      "286 0.07775431871414185\n",
      "287 0.07449959218502045\n",
      "288 0.07136449962854385\n",
      "289 0.06833744049072266\n",
      "290 0.06546807289123535\n",
      "291 0.06271618604660034\n",
      "292 0.06007936969399452\n",
      "293 0.05752433091402054\n",
      "294 0.05512120947241783\n",
      "295 0.05279706418514252\n",
      "296 0.05058007687330246\n",
      "297 0.04845741391181946\n",
      "298 0.04641975089907646\n",
      "299 0.04448764771223068\n",
      "300 0.042622681707143784\n",
      "301 0.0408233217895031\n",
      "302 0.03911640867590904\n",
      "303 0.03749038279056549\n",
      "304 0.03590737655758858\n",
      "305 0.0344102680683136\n",
      "306 0.03295702487230301\n",
      "307 0.03159211575984955\n",
      "308 0.030271874740719795\n",
      "309 0.029011322185397148\n",
      "310 0.027776561677455902\n",
      "311 0.026631439104676247\n",
      "312 0.025538964197039604\n",
      "313 0.024474434554576874\n",
      "314 0.023445429280400276\n",
      "315 0.02248254045844078\n",
      "316 0.02153889462351799\n",
      "317 0.020633818581700325\n",
      "318 0.019790668040513992\n",
      "319 0.01895766332745552\n",
      "320 0.0181729756295681\n",
      "321 0.01743251644074917\n",
      "322 0.01671796850860119\n",
      "323 0.01603940688073635\n",
      "324 0.01537072192877531\n",
      "325 0.014744071289896965\n",
      "326 0.014143739826977253\n",
      "327 0.013555698096752167\n",
      "328 0.01300063356757164\n",
      "329 0.012472403235733509\n",
      "330 0.011962976306676865\n",
      "331 0.011470001190900803\n",
      "332 0.011005043983459473\n",
      "333 0.010555950924754143\n",
      "334 0.010132077150046825\n",
      "335 0.009712629951536655\n",
      "336 0.00932187121361494\n",
      "337 0.008951105177402496\n",
      "338 0.008584248833358288\n",
      "339 0.00824393518269062\n",
      "340 0.007914697751402855\n",
      "341 0.007600391749292612\n",
      "342 0.007290233392268419\n",
      "343 0.007005814928561449\n",
      "344 0.006729997228831053\n",
      "345 0.006454719230532646\n",
      "346 0.006202559918165207\n",
      "347 0.005950086750090122\n",
      "348 0.005720430053770542\n",
      "349 0.005494130775332451\n",
      "350 0.0052802469581365585\n",
      "351 0.0050741760060191154\n",
      "352 0.004877904895693064\n",
      "353 0.004688955377787352\n",
      "354 0.004505170043557882\n",
      "355 0.004330749623477459\n",
      "356 0.00416583800688386\n",
      "357 0.00400713924318552\n",
      "358 0.003849332220852375\n",
      "359 0.003704060334712267\n",
      "360 0.0035640494897961617\n",
      "361 0.0034291595220565796\n",
      "362 0.003299931064248085\n",
      "363 0.0031721130944788456\n",
      "364 0.003052478889003396\n",
      "365 0.0029395290184766054\n",
      "366 0.0028302466962486506\n",
      "367 0.002725373487919569\n",
      "368 0.002623047446832061\n",
      "369 0.0025314060039818287\n",
      "370 0.0024389647878706455\n",
      "371 0.0023478232324123383\n",
      "372 0.002263550413772464\n",
      "373 0.002181108109652996\n",
      "374 0.0021030339412391186\n",
      "375 0.002029406139627099\n",
      "376 0.0019578563515096903\n",
      "377 0.0018893289379775524\n",
      "378 0.0018232528818771243\n",
      "379 0.0017612924566492438\n",
      "380 0.0017000788357108831\n",
      "381 0.0016422735061496496\n",
      "382 0.0015884122112765908\n",
      "383 0.0015329563757404685\n",
      "384 0.0014803230296820402\n",
      "385 0.001430435455404222\n",
      "386 0.0013839001767337322\n",
      "387 0.0013374832924455404\n",
      "388 0.0012930650264024734\n",
      "389 0.001250593806616962\n",
      "390 0.001209931680932641\n",
      "391 0.0011684272903949022\n",
      "392 0.0011314844014123082\n",
      "393 0.0010942848166450858\n",
      "394 0.0010589007288217545\n",
      "395 0.0010233696084469557\n",
      "396 0.0009907938074320555\n",
      "397 0.0009602423524484038\n",
      "398 0.0009312744368799031\n",
      "399 0.000902043713722378\n",
      "400 0.0008744238875806332\n",
      "401 0.0008474811911582947\n",
      "402 0.0008230656967498362\n",
      "403 0.0007979675428941846\n",
      "404 0.0007733285892754793\n",
      "405 0.0007500681094825268\n",
      "406 0.0007278387201949954\n",
      "407 0.00070710270665586\n",
      "408 0.0006858823471702635\n",
      "409 0.000666295934934169\n",
      "410 0.0006465103360824287\n",
      "411 0.0006291908212006092\n",
      "412 0.0006099773454479873\n",
      "413 0.0005930277402512729\n",
      "414 0.0005767522961832583\n",
      "415 0.0005588251515291631\n",
      "416 0.000545336224604398\n",
      "417 0.0005304857040755451\n",
      "418 0.0005158850690349936\n",
      "419 0.000502734153997153\n",
      "420 0.00048822950338944793\n",
      "421 0.0004750542575493455\n",
      "422 0.0004630640905816108\n",
      "423 0.00045027295709587634\n",
      "424 0.0004385693755466491\n",
      "425 0.0004272141377441585\n",
      "426 0.00041628480539657176\n",
      "427 0.0004051158030051738\n",
      "428 0.00039430949254892766\n",
      "429 0.00038415860035456717\n",
      "430 0.00037516187876462936\n",
      "431 0.0003661187074612826\n",
      "432 0.00035712268436327577\n",
      "433 0.0003483204054646194\n",
      "434 0.00033927030744962394\n",
      "435 0.0003307193692307919\n",
      "436 0.00032258121063932776\n",
      "437 0.00031558878254145384\n",
      "438 0.0003074694250244647\n",
      "439 0.00030068380874581635\n",
      "440 0.00029359542531892657\n",
      "441 0.000286687514744699\n",
      "442 0.0002798698260448873\n",
      "443 0.0002737763279583305\n",
      "444 0.0002674416755326092\n",
      "445 0.00026139800320379436\n",
      "446 0.0002551034849602729\n",
      "447 0.0002495546650607139\n",
      "448 0.00024327913706656545\n",
      "449 0.00023805593082215637\n",
      "450 0.00023242083261720836\n",
      "451 0.00022778262791689485\n",
      "452 0.00022282170539256185\n",
      "453 0.00021827928139828146\n",
      "454 0.00021347934671211988\n",
      "455 0.00020916781795676798\n",
      "456 0.0002048318856395781\n",
      "457 0.00020055942877661437\n",
      "458 0.00019642240658868104\n",
      "459 0.0001920047216117382\n",
      "460 0.0001883159566204995\n",
      "461 0.0001843693171394989\n",
      "462 0.00018092089158017188\n",
      "463 0.00017720035975798965\n",
      "464 0.00017380493227392435\n",
      "465 0.00017043434490915388\n",
      "466 0.00016693926590960473\n",
      "467 0.00016356755804736167\n",
      "468 0.0001605146680958569\n",
      "469 0.00015730799350421876\n",
      "470 0.00015439449634868652\n",
      "471 0.0001517595665063709\n",
      "472 0.00014863059914205223\n",
      "473 0.00014612781524192542\n",
      "474 0.00014296620793174952\n",
      "475 0.0001403517962899059\n",
      "476 0.00013760507863480598\n",
      "477 0.00013545292313210666\n",
      "478 0.0001326389901805669\n",
      "479 0.00013061214121989906\n",
      "480 0.00012814224464818835\n",
      "481 0.00012568874808494002\n",
      "482 0.000123246805742383\n",
      "483 0.00012147058441769332\n",
      "484 0.00011961492418777198\n",
      "485 0.00011726325465133414\n",
      "486 0.00011517661914695054\n",
      "487 0.00011334325972711667\n",
      "488 0.00011117944086436182\n",
      "489 0.000109431057353504\n",
      "490 0.0001076505213859491\n",
      "491 0.00010572847531875595\n",
      "492 0.00010380327148595825\n",
      "493 0.00010231318447040394\n",
      "494 0.0001005472950055264\n",
      "495 9.887212218018249e-05\n",
      "496 9.725704876473173e-05\n",
      "497 9.576406591804698e-05\n",
      "498 9.403088188264519e-05\n",
      "499 9.27607252378948e-05\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "dtype = torch.float\n",
    "device = torch.device(\"cpu\")\n",
    "# device = torch.device(\"cuda:0\") # 如果有GPU可以注释掉这行\n",
    "\n",
    "# N是batch size；D_in是输入大小\n",
    "# H是隐层的大小；D_out是输出大小。\n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "\n",
    "# 创建随机的Tensor作为输入和输出\n",
    "# 输入和输出需要的requires_grad=False(默认，因为我们不需要计算loss对它们的梯度。\n",
    "x = torch.randn(N, D_in, device=device, dtype=dtype)\n",
    "y = torch.randn(N, D_out, device=device, dtype=dtype)\n",
    "\n",
    "# 创建weight的Tensor，需要设置requires_grad=True \n",
    "w1 = torch.randn(D_in, H, device=device, dtype=dtype, requires_grad=True)\n",
    "w2 = torch.randn(H, D_out, device=device, dtype=dtype, requires_grad=True)\n",
    "\n",
    "learning_rate = 1e-6\n",
    "for t in range(500):\n",
    "    # Forward阶段: mm实现矩阵乘法，但是它不支持broadcasting。如果需要broadcasting，可以使用matmul\n",
    "    # clamp本来的用途是把值clamp到指定的范围，这里实现ReLU。 \n",
    "    y_pred = x.mm(w1).clamp(min=0).mm(w2)\n",
    "\n",
    "    # pow(2)实现平方计算。 \n",
    "    # loss.item()得到这个tensor的值。也可以直接打印loss，这会打印很多附加信息。\n",
    "    loss = (y_pred - y).pow(2).sum()\n",
    "    print(t, loss.item())\n",
    "\n",
    "    # 使用autograd进行反向计算。它会计算loss对所有对它有影响的requires_grad=True的Tensor的梯度。\n",
    "    \n",
    "    loss.backward()\n",
    "\n",
    "    # 手动使用梯度下降更新参数。一定要把更新的代码放到torch.no_grad()里\n",
    "    # 否则下面的更新也会计算梯度。后面我们会使用torch.optim.SGD，它会帮我们管理这些用于更新梯度的计算。\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        w1 -= learning_rate * w1.grad\n",
    "        w2 -= learning_rate * w2.grad\n",
    "\n",
    "        # 手动把梯度清零 \n",
    "        w1.grad.zero_()\n",
    "        w2.grad.zero_()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py3.6-env",
   "language": "python",
   "name": "py3.6-env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
